#!/usr/bin/python
# _*_ coding: utf-8 _*_

"""
计算生成样本的性能指标
"""
import json
import rouge_score

total_qa_samples = 0
equal_count = 0
answered_correct = 0
with open('fill_blanks_qa_samples.json', encoding='utf8') as fin:
    for line in fin:
        sample = json.loads(line)

        content = sample['文本上下文']
        qas = sample['填空题']

        # 计算准确率
        for qa in qas:
            total_qa_samples += 1
            question = qa['问题']
            answer = qa['答案']
            llm_answer = qa['答题']

            if isinstance(llm_answer, str) or isinstance(llm_answer, int) or isinstance(llm_answer, float):
                llm_answer = str(llm_answer)
                if ' ' in llm_answer:
                    llm_answer = llm_answer.split(' ')
                else:
                    llm_answer = [llm_answer]

            if isinstance(answer, str) or isinstance(answer, int) or isinstance(answer, float):
                answer = str(answer)
                if ' ' in answer:
                    answer = str(answer).split(' ')
                else:
                    answer = [str(answer)]

            # 答案数量匹配
            answer = list(map(str, answer))
            llm_answer = list(map(str, llm_answer))

            if len(answer) == len(llm_answer):
                equal_count += 1

                if '、'.join(answer) == '、'.join(llm_answer):
                    answered_correct += 1
                else:
                    print('=============== 答案不对=============== ')
                    print("上下文：", content)
                    print("问题:", question)
                    print("出题答案：", answer)
                    print("模型答题：", llm_answer)
                    print()
            else:
                print('=============== 答案数量不统一=============== ')
                print("上下文：", content)
                print("问题:", question)
                print("出题答案：", answer)
                print("模型答题：", llm_answer)
                print()

print('填空题答案准确率统计')
print("样本数量：", total_qa_samples)
print("填空题数量准确率：", equal_count / total_qa_samples)
print("填空题答题准确率：", answered_correct / total_qa_samples)
